package com.shujia.core

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Demo7ReduceByKey {
  def main(args: Array[String]): Unit = {
    //1、创建spark环境
    //创建配置文件对象
    val conf = new SparkConf()
    //指定spark执行默认，local：本地执行
    conf.setMaster("local")
    //spark 任务名
    conf.setAppName("wc")
    //创建spark上下文对象
    val sc = new SparkContext(conf)

    //2、读取数据
    //RDD:弹性的分布式数据集
    val lines: RDD[String] = sc.textFile("spark/data/words.txt")

    //3、将一行转换成多行
    val words: RDD[String] = lines.flatMap(line => line.split(","))

    //4、转换成kv格式
    val kvs: RDD[(String, Int)] = words.map(word => (word, 1))

    /**
     * reduceByKey: 对相同key的value进行聚合计算
     * reduceByKey会在map端对数据进行预聚合
     */

    val countRDD: RDD[(String, Int)] = kvs.reduceByKey((x, y) => {
      println(s"x=$x,y=$y")
      x + y
    })

    countRDD.foreach(println)
  }
}
